package com.datacow.moudle.service;

import com.alibaba.fastjson.JSONObject;
import com.datacow.common.webmagic.HttpClientDownloader;
import com.datacow.moudle.constant.BasePageProcessor;
import com.datacow.moudle.constant.ProductHuntInfo;
import com.datacow.moudle.domain.dto.ProductHuntEnAddDTO;
import com.datacow.util.MatchFilterUtil;
import com.datacow.util.Md5Utils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.*;

/**
 * @Description:
 * @Author: 李德森
 * @Date: 2021/1/15
 */
@Service
@Slf4j
public class ProductHuntProcessorService extends BasePageProcessor {


    @Autowired
    ProductHuntService ProductHuntService;

    /**
     * 开始爬取信息并保存的标志
     */
    private int flag = 0;

    /**
     * @Description: process
     * @Param: page
     * @return: void
     * @Date: 2021/1/16
     */
    @Override
    public void process(Page page) {
        flag++;
        List<String> getInterLinkage = getInterLinkageStart(page);
        // 获取所有的链接
        for (String urlRequest : getInterLinkage) {
            //从页面发现后续的url地址来抓取
            page.addTargetRequest(urlRequest);
        }
        /**
         * 开始爬取信息并保存的标志
         */
        if (flag >= 2) {
            saveProductHuntInfo(page);
        }

    }

    /**
     * @Description: 解析详情信息
     * @Param: page
     * @return:
     * @Author: 李德森
     * @Date: 2021/1/15
     */
    @Transactional(rollbackFor = Exception.class)
    public void saveProductHuntInfo(Page page) {
        Map<Integer, String> pictureLinksMap = new HashMap<>();
        //解析页面
        //1、获取标题String
        String title_en = page.getHtml().css(ProductHuntInfo.MatchingRule.CSS_RULE_GET_TITLE).xpath(ProductHuntInfo.MatchingRule.XPATH_RULE_GET_TITLE).toString();
        //过滤Emoji表情
        String title = MatchFilterUtil.removeEmoji(title_en).trim();
        //2、获取url
        String url = page.getUrl().toString();
        //3、获取getIt重定向url
        String getUrlString = page.getHtml().css(ProductHuntInfo.MatchingRule.CSS_RULE_GET_REDIRECTURL).xpath(ProductHuntInfo.MatchingRule.XPATH_RULE_GET_REDIRECTURL).toString();
        String getItUrl = new StringBuffer(ProductHuntInfo.MatchingRule.HOME_PAGE).append(getUrlString).toString();
        //4、获取点赞数
        String likes_en = page.getHtml().css(ProductHuntInfo.MatchingRule.CSS_RULE_GET_LIKES).xpath(ProductHuntInfo.MatchingRule.XPATH_RULE_GET_LIKES).toString();
        //5、获取应用程序详细描述
        String description_en = page.getHtml().css(ProductHuntInfo.MatchingRule.CSS_RULE_GET_DESCRIPTION).nodes().toString();
        //解析数据封装成json格式
        description_en = MatchFilterUtil.getDescriptionEnJson(ProductHuntInfo.MatchingRule.FILTER_TAGS, description_en);
        //6、获取图片链接
        String picture_and_videos_en = page.getHtml().css(ProductHuntInfo.MatchingRule.CSS_RULE_GET_PICTURE_LINKS).xpath(ProductHuntInfo.MatchingRule.XPATH_RULE_GET_PICTURE_LINKS).nodes().toString();
        String[] split = picture_and_videos_en.substring(1, picture_and_videos_en.length() - 1).split(",");
        for (int i = 0; i < split.length; i++) {
            pictureLinksMap.put(i + 1, split[i]);
        }
        //把图片数据封装成json
        String pictureLinksMapToJson = JSONObject.toJSON(pictureLinksMap).toString();
        //当前爬取页面信息爬取结束，将当前页面设置为“跳过”，下次再加入爬虫队列，直接过滤，提高爬取效率。
        page.setSkip(true);

        ProductHuntEnAddDTO productHuntEnAddDTO = new ProductHuntEnAddDTO();
        //通过查询MD5加密url的值---查询url
        ProductHuntEnAddDTO selectByMd5UrlEntity = ProductHuntService.selectByMd5Url(Md5Utils.string2MD5(url)).getData();
        //如果取不到数据就新增，取到就更新
        if (selectByMd5UrlEntity == null) {
            productHuntEnAddDTO.setTitleEn(title);
            productHuntEnAddDTO.setLinkEn(url);
            productHuntEnAddDTO.setRedirectLinkEn(getItUrl);
            productHuntEnAddDTO.setLikesEn(Integer.parseInt(likes_en));
            productHuntEnAddDTO.setDescriptionEn(description_en);
            productHuntEnAddDTO.setMd5Url(Md5Utils.string2MD5(url));
            productHuntEnAddDTO.setPictureAndVideosEn(pictureLinksMapToJson);
            productHuntEnAddDTO.setCreateTime(new Date());
            productHuntEnAddDTO.setUpdateTime(new Date());
            ProductHuntService.addClimeData(productHuntEnAddDTO);
        } else {
            String md5Url = selectByMd5UrlEntity.getLinkEn();
            Long idEn = selectByMd5UrlEntity.getId();
            if (md5Url.equals(url)) {
                productHuntEnAddDTO.setId(idEn);
                productHuntEnAddDTO.setUpdateTime(new Date());
                productHuntEnAddDTO.setLikesEn(Integer.parseInt(likes_en));
                ProductHuntService.updateClimeData(productHuntEnAddDTO);
            }
        }
            log.info("{}","\n#####################加载到的数据######################\n" +
                "#标题:" + title + "\n" +
                "#url:" + url + "\n" +
                "#重定向url:" + getItUrl + "\n" +
                "#点赞数量：" + likes_en + "\n" +
                "#应用程序的描述：" + description_en + "\n" +
                "#获取图片链接：" + pictureLinksMapToJson + "\n" +
                "######################################################");
    }

    /**
     * @Description: 获取链接列表集合的方法
     * @Param: page
     * @return: list
     * @Date: 2021/1/16
     */

    public List<String> getInterLinkageStart(Page page) {
        List<String> urlList = new ArrayList<>();
        //开起--遍历爬取网站信息
        for (Selectable node : page.getHtml().css(ProductHuntInfo.MatchingRule.CSS_RULE_GET_URL).nodes()) {
            //抽取链接
            String interLinkage = node.xpath(ProductHuntInfo.MatchingRule.XPATH_RULE_GET_URL).toString();
            if (node.xpath(ProductHuntInfo.MatchingRule.XPATH_RULE_GET_URL)
                    .regex(ProductHuntInfo.MatchingRule.AFTER_PAGE_RULE).match()) {
                //1、拼接后续的url地址
                String linkAddress = new StringBuffer(ProductHuntInfo.MatchingRule.HOME_PAGE).append(interLinkage).toString();
                urlList.add(linkAddress);
            }

        }
        return urlList;
    }

    /**
     * @Description: 开始爬数据
     * @Param: pageProcessor
     * @return: void
     * @Date: 2021/1/16
     */
    public void start(PageProcessor pageProcessor) {
        //每次开启从零开始计数链接条数，为了从第二条消息开始取值
        flag = 0;
        Spider.create(pageProcessor)
                .setDownloader(new HttpClientDownloader())
                //添加首页url
                .addUrl(ProductHuntInfo.MatchingRule.HOME_PAGE)
                .thread(5).run();
    }

}